package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"
)

var rateLimiter = time.Tick(10 * time.Millisecond)

func Fetch(url string) ([]byte, error) {
	<-rateLimiter

	// 添加user-agent、cookie反爬
	newUrl := strings.Replace(url, "http://", "https://", 1)
	request, err := http.NewRequest(http.MethodGet, newUrl, nil)
	if err != nil {
		return nil, err
	}
	request.Header.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36")
	cookie := "sid=e2e44eaf-5e77-49bb-9ed0-e5bfd6995b35; ec=HjByzbmw-1632610375020-6bfb26a45490d1550070755; FSSBBIl1UgzbN7NO=5TzagWU4ml4QmV3Skzs_gARD5eRrxU35F0JDrz8uV45CJmiDcZf.E0P5uDa2lHIlvrk3G5WSSbiumAttdP_.AYA; Hm_lvt_2c8ad67df9e787ad29dbd54ee608f5d2=1632610361,1632696775,1632841738,1633394703; Hm_lpvt_2c8ad67df9e787ad29dbd54ee608f5d2=1633395668; _efmdata=yvk%2Bvz1tAC8hPhai%2BbyQXTXsRe78gHUSWN7P9RDrRPSKayecNcir9uHUUqqKNmDPRrWceaqslU0EyXbumahzQA8Rqn7QjoZdxLMLy9WH0Vw%3D; _exid=BmKLYGVEy2B%2Fp5CUKNo%2BGsT5F0oVQix6SwAI1u3ejLk2hI%2F873JlJmtUMx4VvHOhm4vnD8ZoRX%2BOwTOPHzi9ow%3D%3D; FSSBBIl1UgzbN7NP=53YXfzKmMLMqqqqmZsIn6nabXPjOX7fNHICfClJDoB0c_n4JmYZ8jaCBJaLZUCiKYcdwAHFU0Q5J1j3n5BnCXdhnlIxKljzYReafLlFv_.Sri1TIgb4UA0xyRf86pByJQtXOcODGh9YKr9yVKb0SzYiWGyEXGIgX.P2bzDqTyGCkpzx_8N3YieYYSBk4rFb3PUveWLDBWC4..4sC6BX4G3HqpFfQMbqts8FIn0BSbc8vGmA9R10ZFkKYCv9kiuPuforxR4qGZQ7aOVCa9JRLKTu"
	request.Header.Add("cookie", cookie)
	client := http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		//return nil, errors.New("xxx") 生成err的两种方法
		return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode)
	}

	bodyReader := bufio.NewReader(resp.Body)
	e := determineEncoding(bodyReader)
	utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())

	return ioutil.ReadAll(utf8Reader)
}

func determineEncoding(r *bufio.Reader) encoding.Encoding {
	bytes, err := r.Peek(1024) // charset.DetermineEncoding()函数里有判断，最多读1024
	if err != nil {
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8 // 默认utf-8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}
